﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.IO.Compression;
using System.Net.Cache;
using com.crawler.common;

namespace com.crawler.components
{
    public class HttpHelper
    {

       

        /// <summary>
        /// 获取页面源代码as
        /// </summary>
        /// <param name="i_url"></param>
        /// <param name="i_post_data"></param>
        /// <returns></returns>
        public static string GetHttpRequestStringByNUll_Post(string i_url)    
        {

            string t_html = string.Empty;

            if (i_url.ToLower().IndexOf("zongheng.com") > -1)
            {
                t_html = GetHttpRequestStringByNUll_PostByProxy(i_url);
            }
            else
            {
                t_html = GetHttpRequestStringByNUll_PostNoProxy(i_url);
            }
           
         
            return t_html;
        }



        /// <summary>
        /// 获取页面源代码
        /// </summary>
        /// <param name="i_url"></param>
        /// <param name="i_post_data"></param>
        /// <returns></returns>
        public static string GetHttpRequestStringByNUll_PostNoProxy(string i_url)
        {

            //默认的编码   
            Encoding t_encoding = null;
            //HttpWebRequest对象用来发起请求   
            HttpWebRequest t_request = null;
            //初始化对像，并设置请求的URL地址       
            t_request = (HttpWebRequest)WebRequest.Create(i_url);

            t_request.ServicePoint.Expect100Continue = false;
            // HttpRequestCachePolicy t_no_cache = new HttpRequestCachePolicy(HttpRequestCacheLevel.NoCacheNoStore);
            // t_request.CachePolicy = t_no_cache;
            t_request.KeepAlive = false;

            //请求方式Get或者Post       
            t_request.Method = "GET";

            //Accept       
            t_request.Accept = "text/html, application/xhtml+xml, */*,zh-CN";
            //ContentType返回类型       
            t_request.ContentType = "text/html";
            //UserAgent客户端的访问类型，包括浏览器版本和操作系统信息        
            t_request.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)";
            //读取数据时的编码方式       
            t_encoding = null;
            //获取影响流的数据对象    
            HttpWebResponse t_response = null;
            //读取流的对象    
            StreamReader t_stream_reader = null;
            //需要返回的数据对象   
            string m_return_data = string.Empty;
            try
            {
                //支持跳转页面，查询结果将是跳转后的页面           
                t_request.AllowAutoRedirect = true;


                ////最大连接数           
                //request.ServicePoint.ConnectionLimit = 1024;           
                #region 得到请求的response
                using (t_response = (HttpWebResponse)t_request.GetResponse())
                {
                    //从这里开始我们要无视编码了               
                    if (t_encoding == null)
                    {
                        MemoryStream t_stream = new MemoryStream();

                        t_stream = GetMemoryStream(t_response.GetResponseStream());



                        byte[] RawResponse = t_stream.ToArray();
                        string temp = Encoding.Default.GetString(RawResponse, 0, RawResponse.Length);
                        //<meta(.*?)charset([\s]?)=[^>](.*?)>                 
                        Match meta = Regex.Match(temp, "<meta([^<]*)charset=([^<]*)[\"']", RegexOptions.IgnoreCase | RegexOptions.Multiline);
                        string t_charter = (meta.Groups.Count > 2) ? meta.Groups[2].Value : string.Empty;

                        t_charter = t_charter.Replace("\"", string.Empty).Replace("'", string.Empty).Replace(";", string.Empty);
                        if (t_charter.Length > 0)
                        {
                            t_encoding = GetSiteEncoding(t_charter);

                        }
                        else
                        {
                            if (t_response.CharacterSet.ToLower().Trim() == "iso-8859-1")
                            {
                                t_encoding = Encoding.GetEncoding("gbk");
                            }
                            else
                            {
                                if (string.IsNullOrEmpty(t_response.CharacterSet.Trim()))
                                {
                                    t_encoding = Encoding.UTF8;
                                }
                                else
                                {
                                    t_encoding = Encoding.GetEncoding(t_response.CharacterSet);
                                }
                            }
                        }
                        m_return_data = t_encoding.GetString(RawResponse);
                    }
                    else
                    {
                        if (t_response.ContentEncoding != null && t_response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                        {
                            //开始读取流并设置编码方式                       
                            using (t_stream_reader = new StreamReader(new GZipStream(t_response.GetResponseStream(), CompressionMode.Decompress), t_encoding))
                            {
                                m_return_data = t_stream_reader.ReadToEnd();
                            }
                        }
                        else
                        {
                            //开始读取流并设置编码方式                        
                            using (t_stream_reader = new StreamReader(t_response.GetResponseStream(), t_encoding))
                            {
                                m_return_data = t_stream_reader.ReadToEnd();
                            }
                        }
                    }
                }
                #endregion
            }
            catch (WebException ex)
            {
                //这里是在发生异常时返回的错误信息  
                LogHelper.Error("获取页面信息错误" + ex.ToString() + "url=" + i_url);
                if (ex.ToString().IndexOf("404") > -1)
                {
                    m_return_data = "404";
                }
                else
                {
                    m_return_data = string.Empty;
                }
                t_response = (HttpWebResponse)ex.Response;
            }

            m_return_data = m_return_data.Replace("\r", "").Replace("\n", "").Replace("\r\n", "");

            return m_return_data;
        }

        /// <summary>
        /// 获取页面源代码
        /// </summary>
        /// <param name="i_url"></param>
        /// <param name="i_post_data"></param>
        /// <returns></returns>
        public static string GetHttpRequestStringByNUll_PostByProxy(string i_url)
        {

            //默认的编码   
            Encoding t_encoding = null;
            //HttpWebRequest对象用来发起请求   
            HttpWebRequest t_request = null;
            //初始化对像，并设置请求的URL地址       
            t_request = (HttpWebRequest)WebRequest.Create(i_url);

            t_request.ServicePoint.Expect100Continue = false;
            // HttpRequestCachePolicy t_no_cache = new HttpRequestCachePolicy(HttpRequestCacheLevel.NoCacheNoStore);
            // t_request.CachePolicy = t_no_cache;
            t_request.KeepAlive = false;

            //请求方式Get或者Post       
            t_request.Method = "GET";

            WebProxy t_web_proxy = ProxyManager.Instance.GetProxy();

            if (t_web_proxy != null)
            {
                t_request.Proxy = t_web_proxy;
            }

            //Accept       
            t_request.Accept = "text/html, application/xhtml+xml, */*,zh-CN";
            //ContentType返回类型       
            t_request.ContentType = "text/html";
            //UserAgent客户端的访问类型，包括浏览器版本和操作系统信息        
            t_request.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)";
            //读取数据时的编码方式       
            t_encoding = null;
            //获取影响流的数据对象    
            HttpWebResponse t_response = null;
            //读取流的对象    
            StreamReader t_stream_reader = null;
            //需要返回的数据对象   
            string m_return_data = string.Empty;
            try
            {
                //支持跳转页面，查询结果将是跳转后的页面           
                t_request.AllowAutoRedirect = true;


                ////最大连接数           
                //request.ServicePoint.ConnectionLimit = 1024;           
                #region 得到请求的response
                using (t_response = (HttpWebResponse)t_request.GetResponse())
                {
                    //从这里开始我们要无视编码了               
                    if (t_encoding == null)
                    {
                        MemoryStream t_stream = new MemoryStream();

                        t_stream = GetMemoryStream(t_response.GetResponseStream());



                        byte[] RawResponse = t_stream.ToArray();
                        string temp = Encoding.Default.GetString(RawResponse, 0, RawResponse.Length);
                        //<meta(.*?)charset([\s]?)=[^>](.*?)>                 
                        Match meta = Regex.Match(temp, "<meta([^<]*)charset=([^<]*)[\"']", RegexOptions.IgnoreCase | RegexOptions.Multiline);
                        string t_charter = (meta.Groups.Count > 2) ? meta.Groups[2].Value : string.Empty;

                        t_charter = t_charter.Replace("\"", string.Empty).Replace("'", string.Empty).Replace(";", string.Empty);
                        if (t_charter.Length > 0)
                        {
                            t_encoding = GetSiteEncoding(t_charter);

                        }
                        else
                        {
                            if (t_response.CharacterSet.ToLower().Trim() == "iso-8859-1")
                            {
                                t_encoding = Encoding.GetEncoding("gbk");
                            }
                            else
                            {
                                if (string.IsNullOrEmpty(t_response.CharacterSet.Trim()))
                                {
                                    t_encoding = Encoding.UTF8;
                                }
                                else
                                {
                                    t_encoding = Encoding.GetEncoding(t_response.CharacterSet);
                                }
                            }
                        }
                        m_return_data = t_encoding.GetString(RawResponse);
                    }
                    else
                    {
                        if (t_response.ContentEncoding != null && t_response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                        {
                            //开始读取流并设置编码方式                       
                            using (t_stream_reader = new StreamReader(new GZipStream(t_response.GetResponseStream(), CompressionMode.Decompress), t_encoding))
                            {
                                m_return_data = t_stream_reader.ReadToEnd();
                            }
                        }
                        else
                        {
                            //开始读取流并设置编码方式                        
                            using (t_stream_reader = new StreamReader(t_response.GetResponseStream(), t_encoding))
                            {
                                m_return_data = t_stream_reader.ReadToEnd();
                            }
                        }
                    }
                }
                #endregion

                ProxyManager.Instance.SetProxy(t_web_proxy);
            }
            catch (WebException ex)
            {
                //这里是在发生异常时返回的错误信息  
                LogHelper.Error("获取页面信息错误" + ex.ToString() + "url=" + i_url);
                if (ex.ToString().IndexOf("404") > -1)
                {
                    m_return_data = "404";
                }
                else
                {
                    m_return_data = string.Empty;
                }
                t_response = (HttpWebResponse)ex.Response;

                if (ex.ToString().Contains("错误的网关"))
                {
                    LogHelper.Debug(string.Format("代理有问题：" + t_web_proxy.Address.Host + ":" + t_web_proxy.Address.Port.ToString()));
                }
            }
           

            m_return_data = m_return_data.Replace("\r", "").Replace("\n", "").Replace("\r\n", "");

            return m_return_data;
        }

    
        private static Encoding GetSiteEncoding(string i_charter)
        {
            i_charter = i_charter.ToLower().Replace("iso-8859-1", "gbk");
            Encoding t_enconding = Encoding.UTF8;
            try
            {
                t_enconding = Encoding.GetEncoding(i_charter);
            }
            catch
            {
                t_enconding = Encoding.UTF8;
            }
            return t_enconding;
        }

        private static MemoryStream GetMemoryStream(Stream i_stream_response)   
        {       
              MemoryStream _stream = new MemoryStream();     
              int Length = 256;      
              Byte[] buffer = new Byte[Length];      
              int bytesRead = i_stream_response.Read(buffer, 0, Length);       
              // write the required bytes        
              while (bytesRead > 0)       
              {          
                  _stream.Write(buffer, 0, bytesRead);          
                  bytesRead = i_stream_response.Read(buffer, 0, Length);     
              }      
              return _stream;  
        }

        public static void DownLoadFile(string i_url, string i_file_name)
        {
            //默认的编码   
            Encoding t_encoding = null;
            //HttpWebRequest对象用来发起请求   
            HttpWebRequest t_request = null;
            //初始化对像，并设置请求的URL地址       
            t_request = (HttpWebRequest)WebRequest.Create(i_url);

            t_request.ServicePoint.Expect100Continue = false;
            // HttpRequestCachePolicy t_no_cache = new HttpRequestCachePolicy(HttpRequestCacheLevel.NoCacheNoStore);
            // t_request.CachePolicy = t_no_cache;
            t_request.KeepAlive = false;

            //请求方式Get或者Post       
            t_request.Method = "GET";
            //Accept       
            t_request.Accept = "text/html, application/xhtml+xml, */*,zh-CN";
            //ContentType返回类型       
            t_request.ContentType = "text/html";
            //UserAgent客户端的访问类型，包括浏览器版本和操作系统信息        
            t_request.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)";
            //读取数据时的编码方式       
            t_encoding = null;
            //获取影响流的数据对象    
            HttpWebResponse t_response = null;

            //需要返回的数据对象   
            string m_return_data = string.Empty;
            try
            {
                //支持跳转页面，查询结果将是跳转后的页面           
                t_request.AllowAutoRedirect = true;


                ////最大连接数           
                //request.ServicePoint.ConnectionLimit = 1024;           
                #region 得到请求的response
                using (t_response = (HttpWebResponse)t_request.GetResponse())
                {
                    //从这里开始我们要无视编码了               
                    if (t_encoding == null)
                    {
                        Stream t_stream = t_response.GetResponseStream(); ;

                        FileStream fs = new FileStream(i_file_name, FileMode.Create, FileAccess.Write);

                        int Length = 256;
                        Byte[] buffer = new Byte[Length];
                        int bytesRead = t_stream.Read(buffer, 0, Length);
                        // write the required bytes        
                        while (bytesRead > 0)
                        {
                            fs.Write(buffer, 0, bytesRead);
                            bytesRead = t_stream.Read(buffer, 0, Length);
                        }      

                        t_stream.Close();
                        fs.Close();

                    }

                }
                #endregion
            }
            catch (WebException ex)
            {
                //这里是在发生异常时返回的错误信息  
                LogHelper.Error("获取页面信息错误" + ex.ToString() + "url=" + i_url);
                m_return_data = string.Empty; ;
                t_response = (HttpWebResponse)ex.Response;
            }
        }


      
    }
}